package rdd.operate;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;

import java.util.Arrays;
import java.util.Iterator;

public class Spark56_WordCount {
    public static void main(String[] args) {
        final SparkConf conf = new SparkConf();
        conf.setMaster("local");
        conf.setAppName("spark");

        final JavaSparkContext jsc = new JavaSparkContext(conf);
        final JavaRDD<String> rdd = jsc.textFile("data/word.txt");

        final JavaRDD<String> words = rdd.flatMap(
                line -> Arrays.asList(line.split(" ")).iterator()
        );

        final JavaPairRDD<String, Iterable<String>> word = words.groupBy(
                wordname -> wordname
        );

        JavaPairRDD<String, Integer> wordcount = word.mapValues(
                itera -> {
                    int num = 0;
                    for (String s : itera) {
                        num++;
                    }
                    return num;
                }
        );

        wordcount.collect().forEach(System.out::println);
        jsc.close();
    }
}
